import sys
import os
import puzautils.table.puza_memory_table as pmt
import puzautils.table.puza_row as pr
import puzautils.table.puza_stream_table as pst
import puzautils.utils.ptime as ptime


C = pst.PuzaStreamTable()
C.OutputFile('./user_profiles_c.db')
D = pst.PuzaStreamTable()
D.OutputFile('./tweets_1_c.db')
user_set = {}
for files in os.listdir('./raw_crawled'):
	if os.path.isdir(files):
		if files.find('rul2_result')!=-1 or files.find('rul3_result')!=-1 or files.find('rul4_result')!=-1:
			dir = os.path.join('./', files)
			sub_file = os.path.join(dir, 'result.db')
			if os.path.exists(sub_file):
				A = pst.PuzaStreamTable()
				A.InputFile(sub_file)
				while True:
					raw_obj = A.StreamInRow()
					if raw_obj != None:
						try:
							r = pr.PuzaRow.Load(raw_obj)
							user_id = r['user']['id']
							if not user_id in user_set:
								user_set[user_id] = pr.PuzaRow(r['user'])
							D.StreamOutRow(r, lformat = 0)
						except:
							continue
					else:
						break
D.StreamClose()
for user_id, user_profile in user_set.iteritems():
	C.StreamOutRow(user_set[user_id], lformat = 0)
C.StreamClose()

